import codecs
import collections

def gen_index_file():
    vocab = load_vocab()
    with codecs.open('./data/ch.index', 'w', 'utf8') as w:
        with codecs.open('./data/ch', 'r', 'utf8') as r:
            for line in r.readlines():
                index_list = []
                for word in line.strip():
                    index_list.append(str(vocab[word]))
                w.write(','.join(index_list))
                w.write('\n')


def load_vocab():
    vocab = collections.OrderedDict()
    with codecs.open('./data/vocab.ch', 'r', 'utf8') as r:
        for line, index in zip(r.readlines(), range(100000)):
            word = line.strip()
            vocab[word] = index
    return vocab

def load_vocab_test():
    vocab = load_vocab()
    print(vocab)

def gen_vocab(in_file, vocab_file):
    vocab = dict()
    with codecs.open(in_file, 'r', 'utf8') as r:
        for line in r.readlines():
            for word in line.strip():
                vocab[word] = 1
    with codecs.open(vocab_file, 'w', 'utf8') as w:
        for word in vocab.keys():
            w.write(word)
            w.write('\n')

if __name__=='__main__':
    #gen_vocab('./data/ch', './data/vocab.ch')
    #load_vocab_test()
    gen_index_file()